package com.attilax.spider;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.attilax.io.filex;

/**
 * Crawling news from hfut news
 *
 * @author hu
 */
public class ByeduJidaoSpiderRss extends BreadthCrawler {

    /**
     * @param crawlPath crawlPath is the path of the directory which maintains
     * information of this crawler
     * @param autoParse if autoParse is true,BreadthCrawler will auto extract
     * links which match regex rules from pag
     */
    public ByeduJidaoSpiderRss(String crawlPath, boolean autoParse) {
        super(crawlPath, autoParse);
        /*start page*/
        this.addSeed("https://zhidao.baidu.com/daily");

        /*ati for addAll   */
        /*fetch url like http://news.hfut.edu.cn/show-xxxxxxhtml*/
     //   this.addRegex("http://news.hfut.edu.cn/show-.*html");
        /*do not fetch jpg|png|gif*/
    //    this.addRegex("-.*\\.(jpg|png|gif).*");
        /*do not fetch url contains #*/
    //    this.addRegex("-.*#.*");
     
    }

    @Override
    public void visit(Page page, CrawlDatums next) {
    	
    	try {
    		  String url = page.getUrl();
    	        /*if page is news page*/
    	      //  if (page.matchUrl("http://news.hfut.edu.cn/show-.*html")) {
    	            /*we use jsoup to parse page*/
    	            Document doc = page.getDoc();

    	            /*extract title and content of news by css selector*/
    	            String title =doc.title();
    	            if(title.length()<7)
    	            {
    	            	System.out.println("--tie too shortL:"+title);
    	            	return;
    	            }
    	            		//page.select("div[id=Article]>h2").first().text();
    	            String content =doc.html(); 
    	            		//page.select("div#artibody", 0).text();

    	            System.out.println("-----------URL:" + url);
    	            System.out.println("-----------title:" + title);
    	       //     System.out.println("content:\n" + content);
    	            
    	            String fname=filex.fileNameEncode(title)+".html";
    	            filex.save(content, "c:\\00spiderSave\\"+fname,"gbk");

    	            /*If you want to add urls to crawl,add them to nextLink*/
    	            /*WebCollector automatically filters links that have been fetched before*/
    	            /*If autoParse is true and the link you add to nextLinks does not match the regex rules,the link will also been filtered.*/
    	        	Elements as = doc.getElementsByTag("a");
    	        	for (Element e : as) {
    	        		String href= "https://zhidao.baidu.com"+ e.attr("href");
    	        		String title2=e.text();
    	        		System.out.println(title2);
    	        		  if(title2.length()<7)
    	    	            {
    	    	            	System.out.println("--tie too shortL:"+title2);
    	    	                 continue;
    	    	            }
    	        		  next.add(href);
    	        	}
    	            
    	          
    	      // }
		} catch (Exception e) {
		e.printStackTrace();
		}
      
    }

    public static void main(String[] args) throws Exception {
        ByeduJidaoSpiderRss crawler = new ByeduJidaoSpiderRss("crawlQc25", true);
        crawler.setThreads(10);
        crawler.setTopN(50);
        //crawler.setResumable(true);
        /*start crawl with depth of 4*/
        crawler.start(2);
        System.out.println("--f");
    }

 
}